#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: copy_company_list.py
@time: 2018/1/6 21:43
"""

import sys

sys.path.append('..')
sys.path.append('../..')
# 网页库
from common import util
from common.mongo import MongDb
from logger import Logger

MONGO_DB_TARGET = {
    "host": "172.16.215.2",
    "port": 40042,
    "db": "company_data",
    "username": "work",
    "password": "haizhi",
}

# 日志模块
log = Logger('copy_gansu_company_list.log').get_logger()

target_db = MongDb(MONGO_DB_TARGET['host'], MONGO_DB_TARGET['port'], MONGO_DB_TARGET['db'],
                   MONGO_DB_TARGET['username'],
                   MONGO_DB_TARGET['password'], log=log)


def main():
    result_list = []
    count = 0
    source_table = 'new_gansu_search_list'
    target_table = 'new_gansu_company_list'
    for item in target_db.traverse_batch_field(source_table, {}, ['_id']):
        company = item.get('_id')
        company = company.replace('%', '')
        company = company.replace('#', '')

        count += 1
        data = {
            '_id': company,
            '_in_time': util.get_now_time(),
        }
        result_list.append(data)
        if len(result_list) >= 500:
            target_db.insert_batch_data(target_table, result_list)
            del result_list[:]

        if count % 1000 == 0:
            log.info("当前数据拷贝进度: count = {}".format(count))

    if len(result_list) > 0:
        target_db.insert_batch_data(target_table, result_list)
        del result_list[:]

    log.info("数据拷贝完成...")


if __name__ == '__main__':
    main()
